Exploring Places¶
Experiments:¶
- Visualizing Places dataset
- Exploring Tags Places
- Exploring Towns & Places Names
- Exploring Properities
- Exploring Descriptions Places Similarities
- Descriptions Places Topic Modelling
In [1]:
import json
import pandas as pd
import plotly.express as px
import os
import plotly.graph_objects as go
import numpy as np
from sentence_transformers import SentenceTransformer
from sklearn.metrics.pairwise import cosine_similarity
from bertopic import BERTopic
In [2]:
#data="places.json"
data="dataset/sample_20200501.json"
with open(data, 'r') as f:
data = json.load(f)
print(len(data["places"]))
places=data["places"]
df = pd.DataFrame(places)
2. Visualizing the places dataframe¶
In [3]:
df["properties"].iloc[0]
Out[3]:
In [4]:
df.shape[0]
Out[4]:
Experiment 1: Exploring Place Ids¶
In [5]:
df_ids=df.groupby(['place_id']).size().reset_index()
df_ids=df_ids.rename(columns={0: "number_of_times"}).sort_values(by=['number_of_times'], ascending=False)
df_ids
Out[5]:
Experiment 2: Exploring Tags Places¶
We are going to separete the elements stored in each tag list into new rows.
In [6]:
df["tags"][0:5]
Out[6]:
In [7]:
df_tags=df.explode('tags')
In [8]:
df_tags
Out[8]:
In [9]:
g_tags=df_tags.groupby(['tags']).size().reset_index()
g_tags=g_tags.rename(columns={0: "number_of_times"}).sort_values(by=['number_of_times'], ascending=False)
g_tags
Out[9]:
In [10]:
px.histogram(g_tags, x="tags", y="number_of_times", histfunc="sum", color="tags", title='Frequency of tags places')
Experiment 3: Exploring Towns & Names¶
In [11]:
df["town"][1:10]
Out[11]:
3.1 Frequency of places grouped by towns¶
In [12]:
df_town=df.dropna(subset=['town'])
town=df_town.groupby(['town']).size().reset_index()
town=town.rename(columns={0: "number_of_times"})
town=town.drop([0])
In [13]:
town=town.sort_values(by=['number_of_times'], ascending=False)
town
Out[13]:
In [14]:
px.scatter(town, x='town', y='number_of_times', color='number_of_times', size="number_of_times", size_max=60, title="Frequency of places grouped by towns")
3.2 Frequency of places grouped by name¶
In [15]:
df_name_town=df.groupby(['name']).size().reset_index()
df_name_town=df_name_town.rename(columns={0: "number_of_times"})
df_name_town=df_name_town.sort_values(by=['number_of_times'], ascending=False)
df_name_town.reset_index()
Out[15]:
3.3. Frequency of places grouped by name and town¶
In [16]:
df_name_town=df.groupby(['name', 'town']).size().reset_index()
df_name_town=df_name_town.rename(columns={0: "number_of_times"})
df_name_town=df_name_town.sort_values(by=['number_of_times'], ascending=False)
df_name_town
Out[16]:
Experiment 4: Exploring Properities¶
In [17]:
df_properties=pd.concat([df.drop(['properties'], axis=1), df['properties'].apply(pd.Series)], axis=1)
In [18]:
df_properties[0:3]
Out[18]:
4.1 Frequency of places grouped by wheelchair-access and town¶
In [19]:
df_properties_wc=df_properties.groupby(['place.facilities.wheelchair-access', 'town']).size().reset_index()
df_properties_wc=df_properties_wc.rename(columns={0: "number_of_times"})
df_properties_wc=df_properties_wc.sort_values(by=['number_of_times'], ascending=False)
df_properties_wc
Out[19]:
4.2 Frequency of places grouped by toilets_disabled and town¶
In [20]:
df_properties_td=df_properties.groupby(['place.facilities.toilets_disabled', 'town']).size().reset_index()
df_properties_td=df_properties_td.rename(columns={0: "number_of_times"})
df_properties_td=df_properties_td.sort_values(by=['number_of_times'], ascending=False)
df_properties_td
Out[20]:
5. Exploring Descriptions¶
In [21]:
df_descriptions=df.explode('descriptions')
df_descriptions=pd.concat([df_descriptions.drop(['descriptions'], axis=1), df_descriptions['descriptions'].apply(pd.Series)], axis=1)
df_descriptions=df_descriptions.dropna(subset=['description']).reset_index()
documents=df_descriptions["description"].values
In [22]:
len(documents)
Out[22]:
In [23]:
import re
from gensim.parsing.preprocessing import remove_stopwords
def clean_documents(text):
text = re.sub(r'\S*@\S*\s?', '', text, flags=re.MULTILINE) # remove email
text = re.sub(r'http\S+', '', text, flags=re.MULTILINE) # remove web addresses
text = re.sub("\'", "", text) # remove single quotes
text = remove_stopwords(text)
return text
In [24]:
d=[]
for text in documents:
d.append(clean_documents(text))
Generating Text Embeddings¶
In [25]:
model = SentenceTransformer('all-MiniLM-L6-v2')
#Training our text_embeddings - using the descriptions available & all-MiniLM-L6-v2 Transformer
text_embeddings = model.encode(d, batch_size = 8, show_progress_bar = True)
In [26]:
np.shape(text_embeddings)
Out[26]:
Description Similarity¶
In [27]:
similarities = cosine_similarity(text_embeddings)
similarities_sorted = similarities.argsort()
id_1 = []
id_2 = []
score = []
for index,array in enumerate(similarities_sorted):
p=len(array)
id_1.append(index)
id_2.append(array[-2])
score.append(similarities[index][array[-2]])
index_df = pd.DataFrame({'id_1' : id_1,
'id_2' : id_2,
'score' : score})
print(index_df)
In [28]:
index_df["score"].sort_values(ascending=False)
Out[28]:
In [29]:
index_df.iloc[85]
Out[29]:
NOTE: Documents 119 and 120 seems to be the most similar. Lets see what they have
In [36]:
documents[119]
Out[36]:
In [37]:
documents[120]
Out[37]:
6. Topic Modelling¶
In [32]:
topic_model = BERTopic(min_topic_size=10).fit(d, text_embeddings)
topics, probs = topic_model.transform(d, text_embeddings)
topic_model.visualize_topics()
In [33]:
topic_model.visualize_barchart()
In [34]:
topic_model.visualize_heatmap()
In [35]:
topic_model.get_topic_freq()
Out[35]: